<?php
/**
 * 将网页处理之后保存起来,等待分词程序解析
 * 计算权重的方法:
 *  网页标题,站点域名的关键词,meta的关键词,网址路径关键词
 */

class CSphider {
    /**
     * 站点名称
     */
    private $site_name;

    /**
     * 起始页面url
     */
    private $root_url;

    /**
     * 索引最大层数
     */
    private $max_floors;

    /**
     * 爬虫是否出站
     */
    private $allow_out_site;

    /**
     * 文件编码
     */
    private $encode;

    /**
     * 域名数组
     */
    private $domain_arr;

    /**
     * links 消息队列
     * @var array
     */
    public $links_queue = array();

    /**
     * links 统计
     */
    private $links_arr = array();

    /**
     * @param $site_name
     * @param $root_url
     * @param int $index_floors
     * @param int $allow_out_site
     * @param string $encode
     * @param int $max_floors
     */
    public function __construct($site_name, $root_url, $index_floors = 1, $allow_out_site = 0, $encode = 'UTF-8',$max_floors = 2)
    {
        $this->site_name = $site_name;
        $this->root_url = $root_url;
        $this->index_floors = $index_floors;
        $this->allow_out_site = $allow_out_site;
        $this->encode = $encode;
        $this->max_floors = $max_floors;
        array_push($this->links_queue,array('url' => $root_url, 'collected' => 0, 'depth' => 0));
        $this->domain_arr = array('.cn','.com','.net','.org','.edu','.gov','.info','.name','.tel');
        $this->run();
    }

    /**
     * 初始化
     */
    public function run()
    {
        if(!$this->site_name || !$this->root_url) {
            return false;
        }

        while( !empty($this->links_queue) ) {
            $tmp = array_shift($this->links_queue);
            //判断不重复
            array_push($this->links_arr,$tmp);
            $this->get_links($tmp);
            unset($tmp);
        }

    }

    /**
     * 获取连接
     *  区分站内/站外
     */
    private function get_links($link)
    {
        if($link['depth'] >= $this->max_floors) {
            return -2; //层次达到
        }

        $url = $link['url'];
        $data = $this->get_content($url);
        $file = $data['file'];
        if($file == 'http://www.php.net/readfile') {
            return false;
        }

        $regs = array ();
        //普通链接
        preg_match_all("/href\s*=\s*[\'\"]?([+:%\/\?~=&;\\\(\),._a-zA-Z0-9-]*)(#[.a-zA-Z0-9-]*)?[\'\" ]?(\s*rel\s*=\s*[\'\"]?(nofollow)[\'\"]?)?/i", $file, $regs, PREG_SET_ORDER);
        foreach($regs as $val) {
            $tmp[]['url'] = $this->get_real_link($url, $val[1]);
            $tmp[end(array_keys($tmp))]['collected'] = 0;
            $tmp[end(array_keys($tmp))]['depth'] = $link['depth']+1;
            $tmp[end(array_keys($tmp))]['children'] = array();

            array_push($this->links_queue,
                array(
                    'url' => $this->get_real_link($url, $val[1]),
                    'collected' => 0,
                    'depth' => $link['depth']+1,
                    'children' => array()
                )
            );

        }

    }

    /**
     * 处理url 绝对路径和相对路径
     * param $p_url 父链接
     * @param $url
     */
    private function get_real_link($p_url, $url)
    {
        if(preg_match('/http(s)?:\/\/([\w-]+\.)+[\w-]+(\/[\w- .\/?%&=]*)?/',$url)) { //标准
            return $url;
        }

        if(strpos($url, 'localhost') > 0) { //主机Host
            return $url;
        }

        $_url = parse_url($p_url);
        $url = str_replace('./','/',$url);
        if(substr($url,0,1) != '/') {
            $url = '/'.$url;
        }

        return $_url['scheme'].'://'.$_url['host'].$url;
    }

    /**
     * 根据url读取文件
     * @param $url
     * @return string
     */
    private function get_content($url)
    {
        ob_start();
        readfile($url);
        $file = ob_get_contents();
        ob_end_clean();
        return array('file'=>$file, 'url'=>$url);
    }


    /**
     * 获取HtmlTitle 权重计算
     */
    public function get_title()
    {

    }

    /**
     * 获取body
     */
    public function get_body()
    {

    }


}